import logging
import os

import scrapy
from bs4 import BeautifulSoup

"""
参考：https://vbni1d7bly.feishu.cn/wiki/FO6CwzrR2iVVm4kcJeGcntGGnhg?fromScene=spaceOverview
"""
class ArticleSpider(scrapy.Spider):
    name = 'article_spider'
    start_urls = ['http://www.ityouknow.com/springcloud/2019/01/26/spring-cloud-gateway-limit.html']

    def parse(self, response):
        try:
            # 检查响应状态码
            if response.status != 200:
                logging.error(f"Failed to fetch page: {response.url}. Status code: {response.status}")
                return

            # 使用BeautifulSoup解析HTML文本
            soup = BeautifulSoup(response.text, 'html.parser')

            # 提取文章标题
            title_tag = soup.find('title')
            # 如果找到标题标签，则提取标题文本
            if title_tag:
                title = title_tag.get_text().strip()
            else:
                title = "标题未找到"  # 如果未找到标题标签，则设置默认标题

            # 提取页面中所有的文字内容
            content = soup.get_text().strip()

            # 打印文章标题和内容
            logging.info("标题: %s", title)
            logging.info("内容: %s", content)

            # 如下直接在根目录输出content.txt文件
            # file_path = 'content.txt'
            # 指定保存路径为当前目录下的 content.txt 文件
            file_path = 'E:/tmp/content.txt'


            # 将文章内容输出为txt文件
            with open(file_path, 'w', encoding='utf-8') as f:
                f.write(content)

            logging.info(f'保存成功: {file_path}')

        except Exception as e:
            logging.error(f'保存失败: {e}')
